﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Text.RegularExpressions;

namespace Microsoft.WindowsAzure.Search
{
    public class Tokenizer
    {
        public static String[] WhitespaceTokenize(String input)
        {
            Regex spaceReg = new Regex("[' ']+");
            return spaceReg.Split(input);
        }

        public static String[] LineTokenize(String input)
        {
            Regex newlineReg = new Regex("\n");
            return newlineReg.Split(input);
        }

        public static String[] CommaTokenize(String input)
        {
            Regex commaReg = new Regex(",");
            return commaReg.Split(input);
        }
    }
}